DOMAIN: Automotive Surveillance.
CONTEXT:Computer vision can be used to automate supervision and generate action appropriate action trigger if the event is predicted from the image of interest. For example a car moving on the road can be easily identified by a camera as make of the car, type, colour, number plates etc.
DATA DESCRIPTION:The Cars dataset contains 16,185 images of 196 classes of cars. The data is split into 8,144 training images and 8,041 testing images, where each class has been split roughly in a 50-50 split. Classes are typically at the level of Make, Model, Year, e.g. 2012 Tesla Model S or 2012 BMW M3 coupe.
Loading Libraries
In [1]:
from sklearn.preprocessing import StandardScaler # StandardScaler is used to resize the distribution of values,
# so that the mean of the observed values is 0 and the standard deviation is 1
import matplotlib.pyplot as plt # Matplotlib is a cross-platform, data visualization and graphical plotting library
import numpy as np # NumPy is a Python library used for working with arrays & linear algebra
import os # The OS module in Python provides functions for creating and removing a directory (folder),
# fetching its contents, changing and identifying the current directory, etc.
import pandas as pd # data processing, in this project used for CSV file I/O (e.g. pd.read_csv)
Importing and Exploring Supporting Data¶
Mounting Google Drive, Data has been downloaded and stored in Google Drive. Using which user can access the supporting files like Annotations, Car Images and Car Names
In [2]:
from google.colab import drive
drive.mount('/content/drive')
Listing the Files and Folders available under Capstone Project
In [42]:
print(os.listdir('/content/drive/MyDrive/CapstoneProject/'))
# listdir() method in python is used to get the list of all files and directories in the specified directory
User Defined Function to create Correlation Plot Matrix
In [43]:
# User-defined function for "Correlation matrix"
def plotCorrelationMatrix(df, graphWidth): # 'df'-dataframe; 'graphwidth'-width (units = inches) are the inputs to this function
"""User Defined function to create Correlation matrix"""
filename = df.dataframeName # the dataframe 'df' name is assigned to 'filename' variable
df = df.dropna('columns') # drop columns with 'NaN'
df = df[[col for col in df if df[col].nunique() > 1]] # keep columns where there are more than 1 unique values
if df.shape[1] < 2: # checking if the dataframe has only one column, in such case no correlation can takes place
print(f'No correlation plots shown: The number of non-NaN or constant columns ({df.shape[1]}) is less than 2')
return
corr = df.corr() # corr() is used to find the pairwise correlation of all columns in the dataframe
plt.figure(num=None, figsize=(graphWidth, graphWidth), dpi=80, facecolor='w', edgecolor='k')
# Create a new figure, or activate an existing figure
# num = A unique identifier for the figure. 'None' = a new figure is created
# figsize = Width, height in inches. Can be (float, float)
# dpi = The resolution of the figure in dots-per-inch.
# facecolor = The background color.
# edgecolor = The border color.
corrMat = plt.matshow(corr, fignum = 1)
# matshow = Display an array as a matrix in a new figure window.
# The origin is set at the upper left hand corner and rows (first dimension of the array) are displayed horizontally.
# corr = The matrix to be displayed.
# fignum = If None, create a new figure window with automatic numbering.
# fignum = If a nonzero integer, draw into the figure with the given number (create it if it does not exist).
# fignum = If 0, use the current axes (or create one if it does not exist).
plt.xticks(range(len(corr.columns)), corr.columns, rotation=90)
# xticks = Get or set the current tick locations and labels of the x-axis
# range(len()) = to access each item by index in columns of corr
# corr.columns = The list of xlabel Text objects
# rotation = rotating xlabels
plt.yticks(range(len(corr.columns)), corr.columns)
# similar to above 'xticks', but with 'y'
plt.gca().xaxis.tick_bottom()
#
plt.colorbar(corrMat)
plt.title(f'Correlation Matrix for {filename}', fontsize=15)
plt.show()
In [44]:
print(plotCorrelationMatrix.__doc__)
In [45]:
# Scatter and density plots
def plotScatterMatrix(df, plotSize, textSize):
"""User Defined function to create Scatterpolt Matrix"""
df = df.select_dtypes(include =[np.number]) # keep only numerical columns
# Remove rows and columns that would lead to df being singular
df = df.dropna('columns')
df = df[[col for col in df if df[col].nunique() > 1]] # keep columns where there are more than 1 unique values
columnNames = list(df)
if len(columnNames) > 10: # reduce the number of columns for matrix inversion of kernel density plots
columnNames = columnNames[:10]
df = df[columnNames]
ax = pd.plotting.scatter_matrix(df, alpha=0.75, figsize=[plotSize, plotSize], diagonal='kde')
corrs = df.corr().values
for i, j in zip(*plt.np.triu_indices_from(ax, k = 1)):
ax[i, j].annotate('Corr. coef = %.3f' % corrs[i, j], (0.8, 0.2), xycoords='axes fraction', ha='center', va='center', size=textSize)
plt.suptitle('Scatter and Density Plot')
plt.show()
In [46]:
print(plotScatterMatrix.__doc__)
Exploring Test Annotation file
Loading TestAnnotation.csv to explore it
In [47]:
nRowsRead = None
# anno_test.csv has 8041 rows in reality, but we are only loading/previewing the first 1000 rows
test_anno_df = pd.read_csv('/content/drive/MyDrive/CapstoneProject/Annotations/TestAnnotation.csv')
test_anno_df.rename(columns={'Bounding Box coordinates': 'x0', 'Unnamed: 2': 'y0','Unnamed: 3': 'x1','Unnamed: 4': 'y1'}, inplace=True)
test_anno_df.dataframeName = 'TestAnnotation.csv'
nRow, nCol = test_anno_df.shape
print(f'There are {nRow} rows and {nCol} columns')
In [48]:
test_anno_df.head()
Out[48]:
Checking Test Annotation files datatypes, there is no issue with the datatypes and every column is mapped properly
In [49]:
test_anno_df.info()
Counting Unique Classes in Test Annotations file
In [54]:
# counting unique values
n = len(pd.unique(test_anno_df['Image class']))
print(n)
TestAnnotation file has around 8041 rows and with 6 columns like Image Name, x0, y0, x1, Y1 and Image Class. all 8041 images has been mapped to 196 different classes. There is no issue with the images¶
In [9]:
plotCorrelationMatrix(test_anno_df, 8)
In [10]:
plotScatterMatrix(test_anno_df, 15, 10)
Exploring Train Annotation file
In [4]:
nRowsRead = None
# anno_test.csv has 8041 rows in reality, but we are only loading/previewing the first 1000 rows
train_anno_df = pd.read_csv('/content/drive/MyDrive/CapstoneProject/Annotations/TrainAnnotations.csv')
train_anno_df.rename(columns={'Bounding Box coordinates': 'x0', 'Unnamed: 2': 'y0','Unnamed: 3': 'x1','Unnamed: 4': 'y1'}, inplace=True)
train_anno_df.dataframeName = 'TrainAnnotation.csv'
nRow, nCol = train_anno_df.shape
print(f'There are {nRow} rows and {nCol} columns')
In [13]:
train_anno_df.head()
Out[13]:
Checking Train Annotation files datatypes, there is no issue with the datatypes and every column is mapped properly
In [17]:
train_anno_df.info()
Counting Unique Classes in Train Annotations file
In [55]:
# counting unique values
n = len(pd.unique(train_anno_df['Image class']))
print(n)
TrainAnnotation file has around 8041 rows and with 6 columns like Image Name, x0, y0, x1, Y1 and Image Class. all 8041 images has been mapped to 196 different classes. There is no issue with the images¶
In [14]:
plotCorrelationMatrix(train_anno_df, 8)
In [15]:
plotScatterMatrix(train_anno_df, 15, 10)
Exploring CarNames file
Loading TrainAnnotation.csv to explore it
In [56]:
nRowsRead = None
#Loading name of cars
carNameDF= pd.read_csv("/content/drive/MyDrive/CapstoneProject/CarNames.csv",header=None)
carNameDF.dataframeName = 'CarNames.csv'
nRow, nCol = carNameDF.shape
print(f'There are {nRow} rows and {nCol} columns')
carNameDF.rename(columns={0: 'CarNames'}, inplace=True)
carNameDF.info()
In [57]:
carNameDF.head()
Out[57]:
Getting unqiue count
In [58]:
n = len(pd.unique(carNameDF['CarNames']))
print(n)
Totally data has more than 196 unique values¶
Map training images to its classes¶
User defined function to create image dataframe
In [19]:
import os
import tensorflow
from keras.preprocessing import image
#from tqdm import tqdm
def createImageDF(imageSourceFolder,target_size_tuple,colormode,totalImagePerClassCap):
dfRows=[]
#l = glob.glob('/content/drive/MyDrive/CapstoneProject/CarImages/TrainImages/*/*')
for dirname,_, filenames in os.walk(imageSourceFolder):
limit=0
if len(filenames) > totalImagePerClassCap or totalImagePerClassCap == 0:
limit=len(filenames)
else:
limit=totalImagePerClassCap
for i in range(0,limit):
processedRows=[]
filename=filenames[i]
processedRows.append(filename)
pathSplits=dirname.split('/')
print(os.path.join(dirname, filename))
imgPath=os.path.join(dirname, filename)
print(imgPath)
img = image.load_img(imgPath)
if target_size_tuple!='':
img_scaled = image.load_img(imgPath, target_size=target_size_tuple, color_mode = colormode)
image_width_scale_fact=target_size_tuple[0]/img.width
image_height_scale_fact=target_size_tuple[1]/img.height
imgArray = image.img_to_array(img_scaled)
else:
img
image_width_scale_fact=img.width
image_height_scale_fact=img.height
#img = Image.open(imgPath)
imgArray = image.img_to_array(img)
#imgArray=np.asarray(img)
processedRows.append(imgArray)
processedRows.append(pathSplits[len(pathSplits)-1])
processedRows.append(image_width_scale_fact)
processedRows.append(image_height_scale_fact)
dfRows.append(processedRows)
return pd.DataFrame(dfRows, columns=["Image Name","image_array","model","image_width_scale_fact","image_height_scale_fact"])
User Defined function to load the single score image
In [20]:
def loadSingleScoringImage(imageSourcePath,target_size_tuple,colormode):
dfRows=[]
processedRow=[]
imgPath=imageSourcePath
print(imgPath)
img = image.load_img(imgPath)
pathSplits=imgPath.split('/')
if target_size_tuple!='':
img_scaled = image.load_img(imgPath, target_size=target_size_tuple, color_mode = colormode)
image_width_scale_fact=target_size_tuple[0]/img.width
image_height_scale_fact=target_size_tuple[1]/img.height
imgArray = image.img_to_array(img_scaled)
else:
img
image_width_scale_fact=img.width
image_height_scale_fact=img.height
#img = Image.open(imgPath)
imgArray = image.img_to_array(img)
#imgArray=np.asarray(img)
processedRow.append(pathSplits[len(pathSplits)-1])
processedRow.append(imgArray)
processedRow.append(pathSplits[len(pathSplits)-2])
processedRow.append(image_width_scale_fact)
processedRow.append(image_height_scale_fact)
dfRows.append(processedRow)
return pd.DataFrame(dfRows, columns=["Image Name","image_array","model","image_width_scale_fact","image_height_scale_fact"])
In [22]:
# import glob
# TrainImage = glob.glob('/content/drive/MyDrive/CapstoneProject/CarImages/TrainImages/*')
# FirstTime = 0
# for ImgFolder in TrainImage:
# try:
# #print(ImgFolder)
# trainImageDF=createImageDF(ImgFolder,'',"rgb",5)
# if FirstTime == 0:
# mergedTrainImageDF = trainImageDF
# FirstTime = FirstTime + 1
# else:
# mergedTrainImageDF = mergedTrainImageDF.append(trainImageDF)
# except:
# print("Success")
#As GPU system got full, I have loaded the sample classes and created Image dataframe
trainImageDF=createImageDF('/content/drive/MyDrive/CapstoneProject/CarImages/TrainImages/Audi 100 Sedan 1994','',"rgb",2)
In [23]:
trainImageDF.shape
Out[23]:
Merging TrainImageDF and Train Annotations
In [24]:
trainImageDF.head()
Out[24]:
In [25]:
train_anno_df.head()
Out[25]:
Merging the trainImageDF and train_anno_df using the common filed - Image Name
Merged Train Image and Train Annonation and Mapped Image classes¶
In [26]:
# Lodaing annotation details for images from train_anno_df i.e. train annotations
trainMergedImageDF=pd.merge(trainImageDF,train_anno_df, on=['Image Name'])
In [27]:
trainMergedImageDF.head()
Out[27]:
Successfully Mapped image classes now, the trainMergedImageDF dataframe has complete information to generate bounding box and for developing model¶
Function to create bounding box on image
In [37]:
import matplotlib.pyplot as plt
import matplotlib.patches as patches
def drawBoxedImages(dataFrame,featureCol,x0,y0,x1,y1,boxlinewidth,boxedgecolor):
fig,ax=plt.subplots(len(dataFrame))
fig.set_size_inches(30,30)
# Display the image
for index, row in dataFrame.iterrows():
data=row[featureCol]
img = image.array_to_img(data)
ax[index].imshow(img)
image_height, image_width, _ = data.shape
rect=[]
rect.append(patches.Rectangle((row[x0], row[y0]), row[x1] - row[x0], row[y1] - row[y0], linewidth=boxlinewidth, edgecolor=boxedgecolor, facecolor='none'))
# Add the patch to the Axes
for l in range(0,len(rect)):
ax[index].add_patch(rect[l])
plt.show()
In [38]:
def drawImages(dataFrame,featureCol):
fig,ax=plt.subplots(len(dataFrame))
fig.set_size_inches(30,30)
# Display the image
for index, row in dataFrame.iterrows():
data=row[featureCol]
img = image.array_to_img(data)
ax[index].imshow(img)
plt.show()
Generating Bounding Box Images for Train data¶
Calling drawBoxedImages function to generate sample bounding box images for training set¶
In [39]:
drawBoxedImages(trainMergedImageDF.head(5),'image_array','x0','y0','x1','y1',2,'r')
Map testing images to its classes¶
In [21]:
testImageDF=createImageDF('/content/drive/MyDrive/CapstoneProject/CarImages/TestImages/Audi 100 Sedan 1994','',"rgb",2)
In [32]:
testImageDF.shape
Out[32]:
Merging the testImageDF and test_anno_df using the common filed - Image Name
In [31]:
testImageDF.head()
Out[31]:
In [33]:
test_anno_df.head()
Out[33]:
In [34]:
# Lodaing annotation details for images from train_anno_df i.e. train annotations
testMergedImageDF=pd.merge(testImageDF,test_anno_df, on=['Image Name'])
Merged Test Image and Test Annonation and Mapped Image classes¶
In [35]:
testMergedImageDF.head(10)
Out[35]:
Generating Bounding Box Images for Test data¶
In [40]:
drawBoxedImages(testMergedImageDF.head(10),'image_array','x0','y0','x1','y1',2,'r')
Building Base Model [ Resnet - PyTorch Framework]¶
Loading required libraries for modelling¶
In [ ]:
import matplotlib.pyplot as plt
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torchvision
import torchvision.models as models
import torchvision.transforms as transforms
import time
import os
import PIL.Image as Image
from IPython.display import display
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
print(torch.cuda.get_device_name(device))
Transforming the images by performing multiple operations like Resizing, Horizontal Flip, Rotations on Train and Test data¶
In [ ]:
dataset_dir = "/content/drive/MyDrive/CapstoneProject/CarImages/"
train_tfms = transforms.Compose([transforms.Resize((400, 400)),
transforms.RandomHorizontalFlip(),
transforms.RandomRotation(15),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
test_tfms = transforms.Compose([transforms.Resize((400, 400)),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
dataset = torchvision.datasets.ImageFolder(root=dataset_dir+"train", transform = train_tfms)
trainloader = torch.utils.data.DataLoader(dataset, batch_size = 32, shuffle=True, num_workers = 2)
dataset2 = torchvision.datasets.ImageFolder(root=dataset_dir+"test", transform = test_tfms)
testloader = torch.utils.data.DataLoader(dataset2, batch_size = 32, shuffle=False, num_workers = 2)
Creating the user defined function for train_model
In [ ]:
def train_model(model, criterion, optimizer, scheduler, n_epochs = 5):
losses = []
accuracies = []
test_accuracies = []
# set the model to train mode initially
model.train()
for epoch in range(n_epochs):
since = time.time()
running_loss = 0.0
running_correct = 0.0
for i, data in enumerate(trainloader, 0):
# get the inputs and assign them to cuda
inputs, labels = data
#inputs = inputs.to(device).half() # uncomment for half precision model
inputs = inputs.to(device)
labels = labels.to(device)
optimizer.zero_grad()
# forward + backward + optimize
outputs = model(inputs)
_, predicted = torch.max(outputs.data, 1)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
# calculate the loss/acc later
running_loss += loss.item()
running_correct += (labels==predicted).sum().item()
epoch_duration = time.time()-since
epoch_loss = running_loss/len(trainloader)
epoch_acc = 100/32*running_correct/len(trainloader)
print("Epoch %s, duration: %d s, loss: %.4f, acc: %.4f" % (epoch+1, epoch_duration, epoch_loss, epoch_acc))
losses.append(epoch_loss)
accuracies.append(epoch_acc)
# switch the model to eval mode to evaluate on test data
model.eval()
test_acc = eval_model(model)
test_accuracies.append(test_acc)
# re-set the model to train mode after validating
model.train()
scheduler.step(test_acc)
since = time.time()
print('Finished Training')
return model, losses, accuracies, test_accuracies
In [ ]:
def eval_model(model):
correct = 0.0
total = 0.0
with torch.no_grad():
for i, data in enumerate(testloader, 0):
images, labels = data
#images = images.to(device).half() # uncomment for half precision model
images = images.to(device)
labels = labels.to(device)
outputs = model_ft(images)
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
test_acc = 100.0 * correct / total
print('Accuracy of the network on the test images: %d %%' % (
test_acc))
return test_acc
Loading Resnet Pretrained model and using it in PyTorch Framework¶
In [ ]:
model_ft = models.resnet34(pretrained=True)
num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, 196)
model_ft = model_ft.to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model_ft.parameters(), lr=0.01, momentum=0.9)
lrscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', patience=3, threshold = 0.9)
Models Keys Settings¶
- Model Fitting - Resnet pretrained model
- Criterion - CrossEntropyLoss
- Optimizer - SGD Optimizer
- lrscheduler - ReduceLROnPlateau
- Epoch - 10
In [ ]:
model_ft, training_losses, training_accs, test_accs = train_model(model_ft, criterion, optimizer, lrscheduler, n_epochs=10)
Plotting Training, Testing Accuracy and Training Loss¶
In [ ]:
# plot the stats
f, axarr = plt.subplots(2,2, figsize = (12, 8))
axarr[0, 0].plot(training_losses)
axarr[0, 0].set_title("Training loss")
axarr[0, 1].plot(training_accs)
axarr[0, 1].set_title("Training acc")
axarr[1, 0].plot(test_accs)
axarr[1, 0].set_title("Test acc")
Out[ ]:
In [ ]:
# tie the class indices to their names
def find_classes(dir):
classes = os.listdir(dir)
classes.sort()
class_to_idx = {classes[i]: i for i in range(len(classes))}
return classes, class_to_idx
classes, c_to_idx = find_classes(dataset_dir+"train")
Testing the base model¶
In [ ]:
# test the model on random images
dataset_dir = "/content/drive/MyDrive/CapstoneProject/CarImages"
torch.save(model_ft,"/content/drive/MyDrive/CapstoneProject/CarImages/working/model.h5")
# switch the model to evaluation mode to make dropout and batch norm work in eval mode
model_ft.eval()
# transforms for the input image
loader = transforms.Compose([transforms.Resize((400, 400)),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
image = Image.open(dataset_dir+"test/Mercedes-Benz C-Class Sedan 2012/01977.jpg")
image = loader(image).float()
image = torch.autograd.Variable(image, requires_grad=True)
image = image.unsqueeze(0)
image = image.cuda()
output = model_ft(image)
conf, predicted = torch.max(output.data, 1)
In [ ]:
# get the class name of the prediction
display(Image.open(dataset_dir+"test/Mercedes-Benz C-Class Sedan 2012/01977.jpg"))
print(classes[predicted.item()], "confidence: ", conf.item())
In [ ]:
dataset_dir = "/content/drive/MyDrive/CapstoneProject/CarImages/test"
In [ ]:
import os
listFolders = os.listdir(dataset_dir)
In [ ]:
import glob
classNames = []
filePaths=[]
for folder in listFolders:
i=0
for file in glob.glob(dataset_dir+"/"+folder+"/*.jpg"):
if(i==5):
break
classNames.append(folder)
filePaths.append(file)
i=i+1
Loading the test image and its prediction¶
In [ ]:
count = 0
for imgPath in filePaths:
# transforms for the input image
try:
loader = transforms.Compose([transforms.Resize((400, 400)),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
image = Image.open(imgPath)
if(image is None):
continue
image = loader(image).float()
image = torch.autograd.Variable(image, requires_grad=True)
image = image.unsqueeze(0)
image = image.cuda()
output = model_ft(image)
conf, predicted = torch.max(output.data, 1)
display(Image.open(imgPath))
print(classes[predicted.item()], "confidence: ", conf.item())
count += 1
if count >= 10:
break
except:
print("Error")
In [ ]: